# load the data
?read.csv
train<-read.csv(file.choose(),stringsAsFactors=F)

str(train)

test<-read.csv(file.choose(),stringsAsFactors=F)
?str
str(test)

#备份
train_org <- train
test_org <- test
library(dplyr)
table(train$Survived) %>%
  prop.table()

#make the first prediction
nrow(test)
test$Survived <- rep(0,418)


#extract PassagerID and Sur information into new container
# and output into a file

submit <- data.frame(PassengerId = test$PassengerId, Survived = test$Survived)
filename <- paste(getwd(),"output","theyallperish.csv",sep = "/")
write.csv(submit,file = filename,row.names = F)


#deeper try with gender and other charactors
train$Sex <- as.factor(train$Sex)
summary(train$Sex) %>%
  prop.table()

#two way comparison
prop.table(table(train$Sex,train$Survived),1)

test$Survived <- 0
test$Survived[test$Sex == "female"] <- 1
str(test)

submit2 <- data.frame(PassengerId = test$PassengerId,Survived = test$Survived)
filename <- paste(getwd(),"output","sexprediction.csv",sep="/")
write.csv(submit2,file=filename,row.names = F)


#digging with age
summary(train$Age)


#之前做决策的变量包含少数几个选项，但是年龄age包含连续的数值，所以这里人为创建一些类别
#小于18岁 child
train$Child <- 0
train$Child[train$Age < 18] <- 1
#child equals 1 means this person is under 18
# NA will fail any boolean test

aggregate(Survived ~ Child + Sex,data = train,FUN = sum)
?aggregate# y~x 依据x 将y 分组，进行FUN 运算
#在本例中，将依据child+sex  对sur 求和 
#结果是：根据年龄 和性别 求出各个类别存活的人 
#求出各个类别的总人数

aggregate(Survived ~ Child + Sex,data=train,FUN = length)
#千呼万唤始出来
aggregate(Survived ~ Child + Sex,data=train,FUN = function(x){sum(x)/length(x)})
#对年龄和性别的分组已经（暂时）不能得出什么新的信息了。下面试着将等级和船票费用加入计算中
str(train$Pclass)
#等级很标准的分为3个类别，好处理
summary(train$Fare)
#好像每个人的票价多少有些不一样，这里将船票分为几个类别：
#小于10,10~20，20~30，大于30
train$Fare2 <- "30+"
train$Fare2[train$Fare < 30 & train$Fare >= 20] <- "20-30"
train$Fare2[train$Fare < 20 & train$Fare >= 10] <- "10-20"
train$Fare2[train$Fare < 10] <- "<10"
?aggregate
a <- aggregate(Survived~ Fare2 + Pclass + Sex, data=train,
          FUN=function(x){sum(x)/length(x)})
a[order(a$Survived),]
#发现几个绝对优势：upper等级的存活率非常高
#船费在20-30的upper女性存活率在83%以上
#船费30+的更是达到了100%
#Middle 等级且船费在10以下的男性全部见阎王了。

#在预测2的基础上
test$Survived <- 0
test$Survived[test$Sex == 'female'] <- 1
mean_of_age <- mean(test$Age,na.rm=T)#求出平均数
test$Age[is.na(test$Age)] <- mean_of_age
#预测生存状况
 test$Survived[test$Fare<10 & test$Sex == 'male'] <- 0 
 test$Survived[test$Fare>30 & test$Pclass==3 & test$Sex == 'female'] <- 0 
 test$Survived[test$Fare<30 & test$Fare>20 & test$Pclass==3 & test$Sex == 'female'] <- 0 
#以上结果不是很理想 原来是传入csvfile的数据框错了、等同于以下操作
  test$Survived[test$Sex == 'female' & test$Pclass == 3 & test$Fare >= 20] <- 0
#生成csv  
submit3 <- data.frame(PassengerId = test$PassengerId,Survived = test$Survived)
filename <- paste(getwd(),"output","mythirdprediction.csv",sep="/")
write.csv(submit3,file=filename,row.names = F)

#决策树
library(rpart)
fit <- rpart(Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked,
             data=train,
             method="class")
#用自带的plot查看树的形状
plot(fit)
text(fit)
#不是很美观，用ggplot等包查看，首先安装包并引入
install.packages('rattle')
install.packages('rpart.plot')
install.packages('RColorBrewer')
library(rattle)
library(rpart.plot)
library(RColorBrewer)

#输出
?fancyRpartPlot

png("kaggle\\output\\dtree.png")
#一个更好的可视化
fancyRpartPlot(fit,sub = "my1stDtree")
dev.off()

#生成答案 
Prediction <- predict(fit, test, type = "class")
submit4 <- data.frame(PassengerId = test$PassengerId,Survived = Prediction)
filename <- paste(getwd(),"kaggle","output","myfirstdtree.csv",sep="/")
write.csv(submit4,file=filename,row.names = F)

?rpart.control


#overfit
overfit <- rpart(Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked,
             data=train,
             method="class", 
             control=rpart.control(minsplit=2, cp=0))

png("kaggle\\output\\overfitdtree.png")
fancyRpartPlot(overfit,sub = "Dtree_overfit")
dev.off()


#根据过度拟合的决策树生成答案 
Prediction <- predict(overfit, test, type = "class")
submit5 <- data.frame(PassengerId = test$PassengerId,Survived = Prediction)
filename <- paste(getwd(),"kaggle","output","overfitdtree.csv",sep="/")
write.csv(submit5,file=filename,row.names = F)

#可交互的决策树
fit <- rpart(Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked,
             data=train,
             method="class",
             control=rpart.control(cp=0))
new.fit <- prp(fit,snip=TRUE)$obj
fancyRpartPlot(new.fit)

train$Name[1]

#为了获得新的变量，并应用到测试集上，需要将测试集和训练集合进行合并。
#合并要求两个或两个以上数据集的字段应当相同。
test$Survived <- NA
combi <- rbind(train, test)

strsplit(combi$Name[1], split='[,.]')

strsplit(combi$Name[1], split='[,.]')[[1]][2]

combi$Title <- sapply(combi$Name, FUN=function(x) {strsplit(x, split='[,.]')[[1]][2]})
#去掉title之前的第一个空格
combi$Title <- sub(' ', '', combi$Title)
table(combi$Title)

combi$Title[combi$Title %in% c('Mme','Mlle')] <- 'Mlle'
#简化分组
combi$Title[combi$Title %in% c('Capt', 'Don', 'Major', 'Sir')] <- 'Sir'
combi$Title[combi$Title %in% c('Dona', 'Lady', 'the Countess', 'Jonkheer')] <- 'Lady'
#convert into factor 
combi$Title <- factor(combi$Title)

#家庭大小
combi$FamilySize <- combi$SibSp + combi$Parch + 1
#姓氏
combi$Surname <- sapply(combi$Name, FUN=function(x) {strsplit(x, split='[,.]')[[1]][1]})
#combine Surname and FamilySize
combi$FamilyID <- paste(as.character(combi$FamilySize), combi$Surname, sep="")

combi$FamilyID[combi$FamilySize <= 2] <- 'Small'
table(combi$FamilyID)
#do some clean job
famIDs <- data.frame(table(combi$FamilyID))
famIDs <- famIDs[famIDs$Freq <= 2,]

#重写那些不正确的familyID，然后将整体转化为因子：
combi$FamilyID[combi$FamilyID %in% famIDs$Var1] <- 'Small'
combi$FamilyID <- factor(combi$FamilyID)
#分割数据
train <- combi[1:891,]
test <- combi[892:1309,]

#生成决策树

fiteng <- rpart(Survived ~ Pclass + Sex + Age + SibSp + Parch + Fare + Embarked + Title + FamilySize + FamilyID,
             data=train, 
             method="class") 
#可视化

png(paste(getwd(),"数据分析第四关","kaggle","output","engdtree.png",sep="/"))
fancyRpartPlot(fiteng,sub = "enged decision tree")
dev.off()

#生成结果
Prediction <- predict(fiteng, test, type = "class")
submit6 <- data.frame(PassengerId = test$PassengerId,Survived = Prediction)
filename <- paste(getwd(),"数据分析第四关","kaggle","output","engdtree.csv",sep="/")
write.csv(submit6,file=filename,row.names = F)

#随机森林

sample(1:10,replace = T)

#观测年龄特征
summary(combi$Age)
#预测年龄
Agefit <- rpart(Age ~ Pclass + Sex + SibSp + Parch + Fare + Embarked + Title + FamilySize,
                  data=combi[!is.na(combi$Age),], 
                  method="anova")
#可视化
fancyRpartPlot(Agefit)



combi$Age[is.na(combi$Age)] <- predict(Agefit, combi[is.na(combi$Age),])


#整体看一下数据还有哪些问题
summary(combi)

which(combi$Embarked == '')
#替换为S并转换为因子
combi$Embarked[which(combi$Embarked == '')] = "S"
combi$Embarked <- factor(combi$Embarked)

#Fare
combi$Fare[which(is.na(combi$Fare))] <- median(combi$Fare, na.rm=TRUE)

#缩减因子数量，将small定义小于3的
combi$FamilyID2 <- combi$FamilyID
combi$FamilyID2 <- as.character(combi$FamilyID2)
combi$FamilyID2[combi$FamilySize <= 3] <- 'Small'
combi$FamilyID2 <- factor(combi$FamilyID2)

summary(combi$FamilyID2)
#随机森林安装包
install.packages('randomForest')
library("randomForest")
set.seed(255)
rnorm(10,mean=0,sd=1)


#分割数据
train <- combi[1:891,]
test <- combi[892:1309,]

#将char变量转化为因子
combi$Sex <- as.factor(combi$Sex)
combi$Name <- as.factor(combi$Name)
combi$Ticket <- as.factor(combi$Ticket)
combi$Cabin <- as.factor(combi$Cabin)
combi$Surname <- as.factor(combi$Surname)
summary(combi$FamilyID) 
forest <- randomForest(as.factor(Survived) ~ Pclass + Sex + Age + SibSp + Parch + Fare +
                                Embarked + Title + FamilySize + FamilyID2,
                              data=train, 
                              importance=TRUE, 
                              ntree=2000)

#查看变量重要性
varImpPlot(forest)

#生成结果
Prediction <- predict(fit, test,type="class")
submit7 <- data.frame(PassengerId = test$PassengerId, Survived = Prediction)
filename <- paste(getwd(),"kaggle","output","randomforest.csv",sep="/") 
write.csv(submit7,file=filename,row.names = F)

#条件推理树
install.packages('party')
library(party)
#设置随机数 
set.seed(128)
preditcfit <- cforest(as.factor(Survived) ~ Pclass + Sex + Age + SibSp + Parch + Fare +
                 Embarked + Title + FamilySize + FamilyID,
               data = train, 
               controls=cforest_unbiased(ntree=2000, mtry=3))
Prediction1 <- predict(preditcfit, test, OOB=TRUE, type = "response")

submit8 <- data.frame(PassengerId = test$PassengerId, Survived = Prediction)
filename <- paste(getwd(),"kaggle","output","conditioninference.csv",sep="/") 
write.csv(submit8,file=filename,row.names = F)
#完
